package com.sena.search;

import com.alibaba.fastjson.JSON;
import com.sena.crawler.SenaCrawler;
import com.sena.crawler.domian.RootLink;
import com.sena.crawler.enums.LinkTypeEnums;
import com.sena.crawler.loader.strategy.HtmlUnitPageLoader;
import com.sena.crawler.util.JedisUtilNew;
import com.sena.domain.SenaClientSite;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.List;

/**
 * @author seven
 * @create 2021-08-10 16:50
 **/
public class SearchFileLink {
    private static Logger logger = LoggerFactory.getLogger(SearchFileLink.class);

    public void SearchSearchFileLinkTask(SenaClientSite site, List<String> blackurls, List<String> whiteurls, List<String> suffixs) {
        //初始化url
        JedisUtilNew jedisUtil = new JedisUtilNew(site.getRedisDbAddr());
        SenaCrawler crawler = new SenaCrawler.Builder()
                .setSenaCrawlerType(LinkTypeEnums.FILE_LINK.getCode())
                .setRootUrls(site.getSiteAddr())
                //设置白名单
                .setWhiteUrlRegexs(whiteurls)
                //设置黑名单
                .setBlackUrlRegexs(blackurls)
                .setPageLoader(new HtmlUnitPageLoader())
                .setThreadCount(site.getThreadNum())
                .setAllowSpread(true)
                .setFilterSuffix(suffixs) //拦截后缀
                .setJdbcUrl("jdbc:mysql://" + site.getMysqlDbIp() + ":" + site.getMysqlDbPort() + "/" + site.getMysqlDbName() + "?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8")
                .setJdbcUserName("root")
                .setJdbcUserPassWord(site.getMysqlDbPassword())
                .setJedisUtil(jedisUtil)
                .build();

        init(jedisUtil, site.getSiteAddr());
        // 启动
        crawler.start(true);
    }


    /**
     * 保存通用url
     */
    public static void init(JedisUtilNew jedisUtil, String link) {

        // 设置根节点
        RootLink rootLink = new RootLink(link, false, null, null, LinkTypeEnums.FILE_LINK.getCode());
        //设置根节点
        jedisUtil.setStringValue(LinkTypeEnums.ROOT_LINK.getCode(), JSON.toJSONString(rootLink));

    }



}
